{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "e0dfb3c5-7419-48b3-ae05-706ec1829b6e",
   "metadata": {},
   "source": [
    "Assumes that the transforms package has been installaed in the venv and all manipulations required for cargo and rep_removal were done in the vnev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "215bd88c-df78-48a8-b43d-3d2168f3cf96",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "%pip install 'data-prep-toolkit-transforms[language]==1.1.1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8d049f72-9ab5-486b-99d0-70e374c9f656",
   "metadata": {},
   "outputs": [],
   "source": [
    "from huggingface_hub import hf_hub_download\n",
    "import pyarrow.parquet as pq\n",
    "import pandas as pd\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ad36252c-8730-46fe-8882-a6be7c5076c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "REPO_ID = \"HuggingFaceFW/fineweb\"\n",
    "FILENAME = \"data/CC-MAIN-2013-20/000_00000.parquet\"\n",
    "file1=hf_hub_download(repo_id=REPO_ID, filename=FILENAME, repo_type=\"dataset\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "90ba29c1-6c70-4fba-b700-8dd2630d8b4e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#os.path.dirname(file1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4204bf13-5af6-4235-9a93-140e181cd3a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "import pyarrow.parquet as pq\n",
    "import pandas as pd\n",
    "table = pq.read_table(file1)\n",
    "table.to_pandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b6bbf09e-240d-4017-9bd3-80c809b01d27",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "from dpk_doc_id import DocID\n",
    "DocID(input_folder= os.path.dirname(file1),\n",
    "        output_folder= \"files-doc-id\",\n",
    "        doc_id_doc_column= \"text\",\n",
    "        doc_id_hash_column= \"document_id\",\n",
    "        doc_id_int_column= \"int_id_column\",\n",
    "        doc_id_start_id= 5).transform()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2b841fe0-696a-47b9-a93d-683190410710",
   "metadata": {},
   "outputs": [],
   "source": [
    "#%%time\n",
    "#import pyarrow.parquet as pq\n",
    "#import pandas as pd\n",
    "#table = pq.read_table('files-doc-id/000_00000.parquet')\n",
    "#table.to_pandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "72d7a18b-a218-4cd2-9877-61cfb32fff1a",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "from dpk_rep_removal import RepRemoval\n",
    "RepRemoval(input_folder= os.path.dirname(file1),\n",
    "            output_folder= \"files-rep_removal\",\n",
    "            rep_removal_contents_column_name='text', \n",
    "            rep_removal_num_threads='1',\n",
    "            ).transform()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "296200e3-503e-4e5f-92f9-4dd78484c615",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "import pyarrow.parquet as pq\n",
    "import pandas as pd\n",
    "table = pq.read_table('files-rep_removal/000_00000.parquet')\n",
    "table.to_pandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e80e2e5a-4318-47bd-a7f0-a446f532e60e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
